/**
 *
 * Copyright Pact Lab of H.I.T.
 *
 * Designed and Implemented by Grid Researching Group, 
 * Pact Lab, Harbin
 * 
 * This Project is part of the national 973 Project:
 * Internet Based Virtual Computing Environment
 *
 * http://pact518.hit.edu.cn
 * 
 * Author:       Meteor <meteorlxk@gmail.com> 
 * Copyright:    pact518 
 * Version:      1.0
 * Created:      2008-11-18 
 * LastModified: 2008-11-18
 */
package edu.hit.pact.pgse.crawler.util;

import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import edu.hit.pact.pgse.crawler.download.WebPageFetcher;

public class RobotFileScanner {
	
	private Log logger;
	
	private WebPageFetcher fetcher;
	
	public RobotFileScanner(WebPageFetcher fetcher) {
		this(LogFactory.getLog(RobotFileScanner.class), fetcher);
	}
	
	public RobotFileScanner(Log logger, WebPageFetcher fetcher) {
		this.logger = logger;
		this.fetcher = fetcher;
	}
	
	public List<String> scan(String hostName){
		List<String> robotsList = new ArrayList<String>();
		
		try {
			URL robotsFileUrl = new URL("http://" + hostName + "/robots.txt");
			
			String robotsContent = fetcher.fetchNormalFile(robotsFileUrl);

			if (robotsContent == null) {
				logger.warn("can't find robots.txt in " + hostName);
				return robotsList;
			}//end if

			StringTokenizer st = new StringTokenizer(robotsContent, "\r\n");
			
			boolean foundUserAgent = false;
			String line;
			while (st.hasMoreTokens()) {
				line = st.nextToken();
				
				if (line.length() != 0) {
					int index = 0;
					if ((index = line.indexOf("User-agent:")) != -1) {
						String userAgent = line.substring(index + "User-agent:".length());
						userAgent = userAgent.trim();
						if (userAgent.equals("*")) {
							foundUserAgent = true;
						} else {
							foundUserAgent = false;
						}
					}//end if
					
					//if foundUserAgent and line begins with "Disallow:"
					if (foundUserAgent && line.startsWith("Disallow:")) {		
						// get disallowed path
						String disallowPath = line.substring("Disallow:".length());

//						// check if there's comment
//						int commentIndex = disallowPath.indexOf("#");
//						if (commentIndex != -1) {
//							// skip comment
//							disallowPath = disallowPath.substring(0, commentIndex);
//						}
						
						disallowPath = disallowPath.trim();
						
						if (disallowPath.length() != 0) {
							robotsList.add(disallowPath);								
						}//end if		
						
					}//end if (foundUserAgent && line.startsWith("Disallow:"))
				}//end if (line.length() != 0)
			}//end while ((line = reader.readLine()) != null)
			
			for (String robots : robotsList) {
				logger.info(hostName + " robots: " + robots);
			}
			
		} catch (Exception e) {
			logger.error(" ERRO: " + e);
		}
		
		return robotsList;
	}
}
